{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "ifmV02zKlsCs"
      },
      "outputs": [],
      "source": [
        "!pip install -qU \\\n",
        "  datasets==2.14.6 \\\n",
        "  cohere==4.34"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q4vTJ-pFmWl5"
      },
      "source": [
        "## Dataset Download"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GFaaDw5VmZEk"
      },
      "source": [
        "We're going to test with a more real world use-case, with messy, imperfect data. We will use the [`jamescalam/ai-arxiv-chunked`](https://huggingface.co/datasets/jamescalam/ai-arxiv-chunked) dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4-FqcdKHmVpa",
        "outputId": "8fadea80-41a3-43f7-d990-d26df6753c0e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['doi', 'chunk-id', 'chunk', 'id', 'title', 'summary', 'source', 'authors', 'categories', 'comment', 'journal_ref', 'primary_category', 'published', 'updated', 'references'],\n",
              "    num_rows: 41584\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "data = load_dataset(\"jamescalam/ai-arxiv-chunked\", split=\"train\")\n",
        "data"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "First we define our embedding function."
      ],
      "metadata": {
        "id": "gp5a_bInyfdX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from getpass import getpass\n",
        "import cohere\n",
        "\n",
        "cohere_key = os.getenv(\"COHERE_API_KEY\") or getpass(\"Cohere API key: \")\n",
        "co = cohere.Client(cohere_key)\n",
        "\n",
        "def embed(docs: list[str]) -> list[list[float]]:\n",
        "    doc_embeds = co.embed(\n",
        "        docs,\n",
        "        input_type=\"search_document\",\n",
        "        model=\"embed-english-v3.0\"\n",
        "    )\n",
        "    return doc_embeds.embeddings"
      ],
      "metadata": {
        "id": "oG6zd1dLw54w"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Use this to build a Numpy array of cohere embedding vectors."
      ],
      "metadata": {
        "id": "1nvrNQSGXvEC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm.auto import tqdm\n",
        "import numpy as np\n",
        "\n",
        "chunks = data[\"chunk\"]\n",
        "batch_size = 128\n",
        "\n",
        "for i in tqdm(range(0, len(chunks), batch_size)):\n",
        "    i_end = min(len(chunks), i+batch_size)\n",
        "    chunk_batch = chunks[i:i_end]\n",
        "    # embed current batch\n",
        "    embed_batch = embed(chunk_batch)\n",
        "    # add to existing np array if exists (otherwise create)\n",
        "    if i == 0:\n",
        "        arr = np.array(embed_batch)\n",
        "    else:\n",
        "        arr = np.concatenate([arr, np.array(embed_batch)])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "8597c42bec234c47808258ce6c3eebeb",
            "fe437081778c4d8fb9a237617dacaf48",
            "227f2196969a4e50b24dd4b2963868d0",
            "c08e68fab52649748ebb0bc11fe3c9cc",
            "1b38114270b444e89a1785a56f6f22fd",
            "4442cd93767744f89f62f2c0557e691a",
            "666c8c95efed48b7b52c0061451f7841",
            "510443a90cba4b50b1583175fff38f79",
            "5f23b85cf2db41de923a663a6d13e935",
            "36e70df2939344639e99a68fe8ace7ac",
            "96fa9d65bdaa4bed88ffb7afb6c61bc8"
          ]
        },
        "id": "EdyWVR17zX7I",
        "outputId": "a85df4bb-d1fa-4e9d-d027-310859abe178"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/325 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8597c42bec234c47808258ce6c3eebeb"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "xq = co.embed(\n",
        "    [\"why should I use llama 2?\"],\n",
        "    input_type=\"search_query\",\n",
        "    model=\"embed-english-v3.0\"\n",
        ").embeddings\n",
        "xq = np.array(xq[0])"
      ],
      "metadata": {
        "id": "cSAbyewJTIgS"
      },
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sim = np.dot(arr, xq.T)\n",
        "top_k=3\n",
        "idx = np.argpartition(sim, -top_k)[-top_k:]\n",
        "idx"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9vqSnuRNTx7p",
        "outputId": "cd304faa-deb4-4f0d-ee2a-be52099208ec"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([18290, 39437, 39445])"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now we need to create the query mechanism, this is simply a cosine similarity calculation between a query vector and our `arr` vectors."
      ],
      "metadata": {
        "id": "Bl9g3ePt029u"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from numpy.linalg import norm\n",
        "\n",
        "# convert chunks list to array for easy indexing\n",
        "chunk_arr = np.array(chunks)\n",
        "\n",
        "def query(text: str, top_k: int=3) -> list[str]:\n",
        "    # create query embedding\n",
        "    xq = co.embed(\n",
        "        [text],\n",
        "        input_type=\"search_query\",\n",
        "        model=\"embed-english-v3.0\"\n",
        "    ).embeddings\n",
        "    xq = np.array(xq[0])\n",
        "    # calculate cosine similarities\n",
        "    sim = np.dot(arr, xq.T)\n",
        "    print(sim.shape)\n",
        "    # get indices of top_k records\n",
        "    idx = np.argpartition(sim, -top_k)[-top_k:]\n",
        "    print(sim[idx])\n",
        "    # get docs and print\n",
        "    docs = chunk_arr[idx]\n",
        "    print(docs.shape)\n",
        "    for d in docs.tolist():\n",
        "        print(d)\n",
        "        print(\"----------\")"
      ],
      "metadata": {
        "id": "MR7WyDiatlsX"
      },
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "query(\"why should I use llama 2?\")"
      ],
      "metadata": {
        "id": "u2NjYxsn7J5f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4cd10eba-7930-4313-86f7-26b00f9715ef"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(41584,)\n",
            "[0.47466855 0.53013851 0.53044737]\n",
            "(3,)\n",
            "\u0003Equal contribution. Correspondence: {htouvron,\n",
            "thibautlav,gizacard,egrave,glample}@meta.com\n",
            "1https://github.com/facebookresearch/llamaperformance, a smaller one trained longer will\n",
            "ultimately be cheaper at inference. For instance,\n",
            "although Hoffmann et al. (2022) recommends\n",
            "training a 10B model on 200B tokens, we ﬁnd\n",
            "that the performance of a 7B model continues to\n",
            "improve even after 1T tokens.\n",
            "The focus of this work is to train a series of\n",
            "language models that achieve the best possible performance at various inference budgets, by training\n",
            "on more tokens than what is typically used. The\n",
            "resulting models, called LLaMA , ranges from 7B\n",
            "to 65B parameters with competitive performance\n",
            "compared to the best existing LLMs. For instance,\n",
            "LLaMA-13B outperforms GPT-3 on most benchmarks, despite being 10 \u0002smaller. We believe that\n",
            "this model will help democratize the access and\n",
            "study of LLMs, since it can be run on a single GPU.\n",
            "At the higher-end of the scale, our 65B-parameter\n",
            "model is also competitive with the best large language models such as Chinchilla or PaLM-540B.\n",
            "Unlike Chinchilla, PaLM, or GPT-3, we only\n",
            "----------\n",
            "Alan Schelten Ruan Silva Eric Michael Smith Ranjan Subramanian Xiaoqing Ellen Tan Binh Tang\n",
            "Ross Taylor Adina Williams Jian Xiang Kuan Puxin Xu Zheng Yan Iliyan Zarov Yuchen Zhang\n",
            "Angela Fan Melanie Kambadur Sharan Narang Aurelien Rodriguez Robert Stojnic\n",
            "Sergey Edunov Thomas Scialom\u0003\n",
            "GenAI, Meta\n",
            "Abstract\n",
            "In this work, we develop and release Llama 2, a collection of pretrained and ﬁne-tuned\n",
            "large language models (LLMs) ranging in scale from 7 billion to 70 billion parameters.\n",
            "Our ﬁne-tuned LLMs, called L/l.sc/a.sc/m.sc/a.sc /two.taboldstyle-C/h.sc/a.sc/t.sc , are optimized for dialogue use cases. Our\n",
            "models outperform open-source chat models on most benchmarks we tested, and based on\n",
            "ourhumanevaluationsforhelpfulnessandsafety,maybeasuitablesubstituteforclosedsource models. We provide a detailed description of our approach to ﬁne-tuning and safety\n",
            "----------\n",
            "asChatGPT,BARD,andClaude. TheseclosedproductLLMsareheavilyﬁne-tunedtoalignwithhuman\n",
            "preferences, which greatly enhances their usability and safety. This step can require signiﬁcant costs in\n",
            "computeandhumanannotation,andisoftennottransparentoreasilyreproducible,limitingprogresswithin\n",
            "the community to advance AI alignment research.\n",
            "In this work, we develop and release Llama 2, a family of pretrained and ﬁne-tuned LLMs, L/l.sc/a.sc/m.sc/a.sc /two.taboldstyle and\n",
            "L/l.sc/a.sc/m.sc/a.sc /two.taboldstyle-C/h.sc/a.sc/t.sc , at scales up to 70B parameters. On the series of helpfulness and safety benchmarks we tested,\n",
            "L/l.sc/a.sc/m.sc/a.sc /two.taboldstyle-C/h.sc/a.sc/t.sc models generally perform better than existing open-source models. They also appear to\n",
            "be on par with some of the closed-source models, at least on the human evaluations we performed (see\n",
            "----------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query(\"can you tell me about red teaming for llama 2?\")"
      ],
      "metadata": {
        "id": "ZSIBTMUy7qc0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5f33ec66-e7f6-41f1-85e5-0dd0a1f4f635"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(41584,)\n",
            "[0.47529661 0.47952211 0.48869599]\n",
            "(3,)\n",
            "the training data [13], aiding in disinformation campaigns [12], generating extremist texts [37], spreading\n",
            "falsehoods [35], and more [9, 10, 18, 57, 22, 51]. As AI systems improve, the scope of possible harms seems\n",
            "likely to grow [22]. Many strategies have been developed to address some of these harms (e.g., [58, 4, 48,\n",
            "36, 34, 19, 60]). One potentially useful tool for addressing harm is red teaming—using manual or automated\n",
            "methods to adversarially probe a language model for harmful outputs, and then updating the model to avoid\n",
            "such outputs [42, 20, 3, 11]. In this paper, we describe our early efforts to implement manual red teaming to\n",
            "both make models safer and measure the safety of our models. The models trained with red team data were\n",
            "described in [4], so here we focus on describing our red team results and techniques in detail in the hope that\n",
            "others may beneﬁt from and improve on them.\n",
            "\u0003Correspondence to: {deep, liane, jackson, jared, jack}@anthropic.com\n",
            "Authors above the line break are core contributors. Author contributions are listed in §A.1.arXiv:2209.07858v2  [cs.CL]  22 Nov 2022\n",
            "2.7B 13B 52B\n",
            "----------\n",
            "including limitations and risks that might be exploited by m alicious actors. Further, existing\n",
            "red teaming approaches are insufﬁcient for addressing thes e concerns in the AI context.\n",
            "In order for AI developers to make veriﬁable claims about the ir AI systems being safe or secure, they need\n",
            "processes for surfacing and addressing potential safety an d security risks. Practices such as red teaming\n",
            "exercises help organizations to discover their own limitat ions and vulnerabilities as well as those of the\n",
            "AI systems they develop, and to approach them holistically , in a way that takes into account the larger\n",
            "environment in which they are operating.23\n",
            "A red team exercise is a structured effort to ﬁnd ﬂaws and vuln erabilities in a plan, organization, or\n",
            "technical system, often performed by dedicated \"red teams\" that seek to adopt an attacker’s mindset\n",
            "and methods. In domains such as computer security , red teams are routinely tasked with emulating\n",
            "attackers in order to ﬁnd ﬂaws and vulnerabilities in organi zations and their systems. Discoveries made\n",
            "by red teams allow organizations to improve security and sys tem integrity before and during deployment.\n",
            "Knowledge that a lab has a red team can potentially improve th e trustworthiness of an organization with\n",
            "----------\n",
            "Red teaming ChatGPT via Jailbreaking:\n",
            "Bias, Robustness, Reliability and Toxicity\n",
            "Terry Yue Zhuo1,2§, Yujin Huang2, Chunyang Chen2, Zhenchang Xing1,3\n",
            "1CSIRO’s Data61\n",
            "2Monash University\n",
            "3Australian National University\n",
            "Warning: this paper may contain content that is offensive or upsetting.\n",
            "Abstract—Recent breakthroughs in natural language processing (NLP) have permitted the synthesis and comprehension\n",
            "of coherent text in an open-ended way, therefore translating\n",
            "the theoretical algorithms into practical applications. The large\n",
            "languagemodels(LLMs)havesignificantlyimpactedbusinesses\n",
            "such as report summarization software and copywriters. Observations indicate, however, that LLMs may exhibit social\n",
            "prejudice and toxicity, posing ethical and societal dangers\n",
            "of consequences resulting from irresponsibility. Large-scale\n",
            "benchmarks for accountable LLMs should consequently be\n",
            "developed. Although several empirical investigations reveal\n",
            "the existence of a few ethical difficulties in advanced LLMs,\n",
            "there is little systematic examination and user study of the\n",
            "risks and harmful behaviors of current LLM usage. To further\n",
            "educate future efforts on constructing ethical LLMs responsibly, we perform a qualitative research method called “red\n",
            "teaming” on OpenAI’s ChatGPT1to better understand the\n",
            "----------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query(\"what is the best llm?\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nhhJOqXTCJN7",
        "outputId": "e56102e5-8ca1-44a0-8119-eec874997550"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(41584,)\n",
            "[0.49388744 0.5080906  0.51699355]\n",
            "(3,)\n",
            "for ﬁtting LLMs is an enormous training dataset, e.g., the Pile [15], which contains documents from\n",
            "Arxiv, PubMed, Stack Exchange, Wikipedia, as well as a subset of Common Crawl2, and GitHub,\n",
            "among others. For these kinds of LLMs, [16] introduced the terminology of foundation models ,\n",
            "which deﬁnes training on a very large data basis and the ability to adapt to a variety of downstream\n",
            "tasks.\n",
            "2.2 ChatGPT\n",
            "ChatGPT is an LLM developed by OpenAI that was ﬁrst released on November 30th, 2022. The\n",
            "user can directly prompt the model via an API in a conversational way, e.g., allowing for follow-up\n",
            "questions or admission of mistakes [1]. The backbone of ChatGPT is based on the generative pretrained transformer series (GPT; [17, 18, 19]). Despite the success and capacity of the third GPT\n",
            "iteration (GPT-3) [19] with 175B parameters, the challenge of engineering text prompts for achieving\n",
            "the desired generative output remained. This is due to the autoregressive training procedure, which\n",
            "tasks the model to predict a token based on the previous text and thus is optimized for text completion\n",
            "and not dialogues. To improve the dialogue capabilities of the model as well as to reduce bias and\n",
            "----------\n",
            "governments, would never do harms with LLMs. Without access to LLMs, we cannot even realize\n",
            "the potential role of LLMs in harms.\n",
            "Thus, an open LLM can provide access and transparency to all researchers, and facilitate the research\n",
            "developments of reducing the potential harms of LLMs, such as algorithms to identify the synthetic\n",
            "text Gehrmann et al. (2019). In addition, it is known that LLMs can suffer from problems in fairness,\n",
            "bias, privacy, and truthfulness Zhang et al. (2021); Lin et al. (2022); Liang et al. (2021); Bender\n",
            "et al. (2021). Thus, instead of providing APIs to black-box models, an open LLM can help reveal\n",
            "the model parameters and internal states corresponding to speciﬁc inputs. In conclusion, an open\n",
            "LLM empowers us to conduct studies on LLMs’ ﬂaws in depth and to improve LLMs in terms of\n",
            "ethical concerns.\n",
            "Ethical Evaluation and Improvements. We evaluate GLM-130B on a wide range of ethical evaluation benchmarks, including bias measurement (Nadeem et al., 2021; Nangia et al., 2020), hate speech\n",
            "----------\n",
            "and as our pilot experiments have demonstrated\n",
            "the effectiveness of the relevance judgments generated by LLMs, we believe it deserves further\n",
            "exploration. (2) Instruction-tuning LLMs for a\n",
            "universal information access system. Instructiontuning LLMs for diverse ranking tasks, such as passage ranking, entity ranking, response ranking, evidence ranking and etc., has great potential toward\n",
            "a more powerful, universal information access system. (3) End-to-end IR model. Existing multi-stage\n",
            "IR systems always follow a “index-retrieve-rank”\n",
            "pipeline, and the separation of different components makes it hard for end-to-end learning. Considering the remarkable performance of LLMs, it’s\n",
            "possible to use only one LLM covering every component in the IR system, such as retrieval and ranking. (4) Improving the efﬁciency of LLMs. Though\n",
            "effective, current LLMs generally have hundreds\n",
            "of billions of parameters, and deploying them to\n",
            "real industrial scenarios is prohibitively expensive.\n",
            "Thus, improving the efﬁciency of LLMs, such as\n",
            "reducing to small models, and lightweight learning,\n",
            "is very worthy of further exploration.\n",
            "References\n",
            "Negar Arabzadeh, Alexandra Vtyurina, Xinyi Yan, and\n",
            "Charles L. A. Clarke. 2021. Shallow pooling for\n",
            "----------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query(\"what is the difference between gpt-4 and llama 2?\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wLE9QgwwCJCT",
        "outputId": "2620dfd5-a950-409e-e64b-33a6cfa4e060"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(41584,)\n",
            "[0.63758657 0.63869209 0.64677286]\n",
            "(3,)\n",
            "to GPT-3 corresponds to the Stanford Alpaca model. From Figure 3(a), we observe that ( i) For the\n",
            "“Helpfulness” criterion, GPT-4 is the clear winner with 54.12% of the votes. GPT-3 only wins 19.74%\n",
            "of the time. ( ii) For the “Honesty” and “Harmlessness” criteria, the largest portion of votes goes\n",
            "to the tie category, which is substantially higher than the winning categories but GPT-3 (Alpaca) is\n",
            "slightly superior.\n",
            "Second, we compare GPT-4-instruction-tuned LLaMA models against the teacher model GPT-4 in\n",
            "Figure 3(b). The observations are quite consistent over the three criteria: GPT-4-instruction-tuned\n",
            "LLaMA performs similarly to the original GPT-4. We conclude that learning from GPT-4 generated\n",
            "5\n",
            "60% 70% 80% 90% 100%12345BRanking Group 94% 624 : 66792% 614 : 67091% 623 : 68289% 597 : 66989% 605 : 67891% 609 : 666\n",
            "----------\n",
            "of the reward model.\n",
            "We compare all the chatbots in Figure 4(c,d). Instruction tuning of LLaMA with GPT-4 often\n",
            "achieves higher performance than tuning with text-davinci-003 (i.e.,Alpaca) and no tuning\n",
            "(i.e.,LLaMA): The 7B LLaMA GPT4 outperforms the 13B Alpaca and LLaMA. However, there is\n",
            "still a gap compared with large commercial chatbots such as GPT-4.\n",
            "We further study the performance of all the chatbots in Chinese in Figure 5. We ﬁrst translate English\n",
            "responses of chatbots into Chinese using GPT-4. We also translate English questions into Chinese to\n",
            "obtain answers with GPT-4. The comparisons against translated and generated Chinese responses\n",
            "from GPT-4 are shown in Figure 5 (a) and (b), respectively. There are two interesting observations: (i)\n",
            "we ﬁnd that the relative score metric of GPT-4 evaluation (Vicuna, 2023) is quite consistent, both in\n",
            "terms of different opponent models ( i.e.,ChatGPT or GPT-4) and languages ( i.e.,English or Chinese).\n",
            "(ii)For GPT-4 results alone, the translated responses show superior performance over the generated\n",
            "----------\n",
            "-0.043\n",
            "-0.009+0.0132-0.004 +0.0562\n",
            "+0.0387-0.012\n",
            "-0.076Alpaca: 0.39 LLaMA-GPT4: 0.34 GPT4: 0.37Figure 6: ROUGE-L on unnatural instructions evaluated with 9K samples. The instructions are\n",
            "grouped into four subsets based on the ground-truth response length. The mean values are reported in\n",
            "the legend. The difference with GPT-4 is reported on the bar per group. LLaMA-GPT4 is a closer\n",
            "proxy to GPT-4 than Alpaca.\n",
            "closely follow the behavior of GPT-4. When the sequence length is short, both LLaMA-GPT4 and\n",
            "GPT-4 can generate responses that contains the simple ground truth answers, but add extra words to\n",
            "make the response more chat-like, which probably leads to lower ROUGE-L scores.\n",
            "5 R ELATED WORK\n",
            "Instruction Tuning. Instruction tuning of LLMs is an increasingly popular research direction in\n",
            "NLP (Zhong et al., 2021; Ouyang et al., 2022; Wei et al., 2021). Existing works aim to improve\n",
            "the quality and scale of three factors in the development pipeline, including instruction-following\n",
            "----------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---"
      ],
      "metadata": {
        "id": "udhnQP3_XdnW"
      }
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8597c42bec234c47808258ce6c3eebeb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_fe437081778c4d8fb9a237617dacaf48",
              "IPY_MODEL_227f2196969a4e50b24dd4b2963868d0",
              "IPY_MODEL_c08e68fab52649748ebb0bc11fe3c9cc"
            ],
            "layout": "IPY_MODEL_1b38114270b444e89a1785a56f6f22fd"
          }
        },
        "fe437081778c4d8fb9a237617dacaf48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4442cd93767744f89f62f2c0557e691a",
            "placeholder": "​",
            "style": "IPY_MODEL_666c8c95efed48b7b52c0061451f7841",
            "value": "100%"
          }
        },
        "227f2196969a4e50b24dd4b2963868d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_510443a90cba4b50b1583175fff38f79",
            "max": 325,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5f23b85cf2db41de923a663a6d13e935",
            "value": 325
          }
        },
        "c08e68fab52649748ebb0bc11fe3c9cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_36e70df2939344639e99a68fe8ace7ac",
            "placeholder": "​",
            "style": "IPY_MODEL_96fa9d65bdaa4bed88ffb7afb6c61bc8",
            "value": " 325/325 [07:04&lt;00:00,  1.34s/it]"
          }
        },
        "1b38114270b444e89a1785a56f6f22fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4442cd93767744f89f62f2c0557e691a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "666c8c95efed48b7b52c0061451f7841": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "510443a90cba4b50b1583175fff38f79": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5f23b85cf2db41de923a663a6d13e935": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "36e70df2939344639e99a68fe8ace7ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96fa9d65bdaa4bed88ffb7afb6c61bc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}